{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "# from dfply import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CVaR Optimization\n",
    "## lognormal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "risk = pickle.load(open(\"cvar/risk_cvar_lognormal.pkl\", \"rb\"))\n",
    "sample_size = list(risk.keys())\n",
    "obj_coefs = list(risk[sample_size[0]].keys())\n",
    "method = list(risk[sample_size[0]][obj_coefs[0]].keys())\n",
    "runs = len(risk[sample_size[0]][obj_coefs[0]][method[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = {\n",
    "    \"n\": [key for key in sample_size for i in range(runs * len(method))],\n",
    "    \"obj_coef\": [\"0\"] * len(sample_size) * len(method) * runs,\n",
    "    \"Method\": [key for key in method for i in range(runs)] * len(sample_size),\n",
    "    \"Constraint\": [\"no\"] * len(sample_size) * len(method) * runs, \n",
    "    \"rel_risk\": [0] * len(sample_size) * len(method) * runs\n",
    "}\n",
    "df = pd.DataFrame(data)\n",
    "type_convert = {\n",
    "    \"n\": str,\n",
    "    \"obj_coef\": str,\n",
    "    \"Method\": str,\n",
    "    \"Constraint\": str, \n",
    "    \"rel_risk\": float\n",
    "}\n",
    "df = df.astype(type_convert) \n",
    "for n in sample_size:\n",
    "    for obj_coef in obj_coefs:\n",
    "        for m in method:\n",
    "#             df.loc[(df.n == n) & (df.Method == m), \"rel_risk\"] = (risk[n][obj_coef][\"SAA\"] - risk[n][obj_coef][m])/(risk[n][obj_coef][\"SAA\"] - risk[n][obj_coef][\"oracle\"])\n",
    "            df.loc[(df.n == n) & (df.Method == m), \"rel_risk\"] = risk[n][obj_coef][m]/risk[n][obj_coef][\"oracle\"]\n",
    "            \n",
    "df = df.loc[df.Method != \"oracle\"]\n",
    "df.loc[df.Method == \"rf_oracle\", \"Constraint\"] = \"yes\"\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Constraint\"] = \"yes\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Constraint\"] = \"yes\"\n",
    "\n",
    "df.loc[df.Method == \"rf_oracle\", \"Method\"] = \"StochOptForest\\n(oracle)\"\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_approx_sol_unconstr\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk_unconstr\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"grf\", \"Method\"] = \"GenRandForest\\n(modified)\"\n",
    "df.loc[df.Method == \"rf_rf\", \"Method\"] = \"RandForest\"\n",
    "df.loc[df.Method == \"rf_random\", \"Method\"] = \"RandSplitForest\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"cvar/risk_lognormal.csv\", index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature importance "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_imp = pickle.load(open(\"cvar/feature_imp_cvar_lognormal.pkl\", 'rb'))\n",
    "temp = feature_imp[\"800\"][\"0\"][0]\n",
    "mean_imp = {key: temp[key].mean(0) for key in temp.keys()}\n",
    "pd.DataFrame({key: (mean_imp[key] - np.min(mean_imp[key]))/(np.max(mean_imp[key]) - np.min(mean_imp[key])) \n",
    "              for key in mean_imp.keys()}).to_csv(\"cvar/feature_imp_cvar_lognormal.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature Splitting Frequency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_splt = pickle.load(open(\"cvar/feature_split_cvar_lognormal.pkl\", 'rb'))\n",
    "temp = feature_splt[\"800\"][\"0\"]\n",
    "mean_imp = {key: temp[key].mean(0) for key in temp.keys()}\n",
    "pd.DataFrame(mean_imp).to_csv(\"cvar/feature_split_cvar_lognormal.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## lognormal oracle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "risk = pickle.load(open(\"cvar/risk_cvar_lognormal_oracle.pkl\", \"rb\"))\n",
    "sample_size = list(risk.keys())\n",
    "obj_coefs = list(risk[sample_size[0]].keys())\n",
    "method = list(risk[sample_size[0]][obj_coefs[0]].keys())\n",
    "runs = len(risk[sample_size[0]][obj_coefs[0]][method[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = {\n",
    "    \"n\": [key for key in sample_size for i in range(runs * len(method))],\n",
    "    \"obj_coef\": [\"0\"] * len(sample_size) * len(method) * runs,\n",
    "    \"Method\": [key for key in method for i in range(runs)] * len(sample_size),\n",
    "    \"Constraint\": [\"no\"] * len(sample_size) * len(method) * runs, \n",
    "    \"rel_risk\": [0] * len(sample_size) * len(method) * runs\n",
    "}\n",
    "df = pd.DataFrame(data)\n",
    "type_convert = {\n",
    "    \"n\": str,\n",
    "    \"obj_coef\": str,\n",
    "    \"Method\": str,\n",
    "    \"Constraint\": str, \n",
    "    \"rel_risk\": float\n",
    "}\n",
    "df = df.astype(type_convert) \n",
    "for n in sample_size:\n",
    "    for obj_coef in obj_coefs:\n",
    "        for m in method:\n",
    "            df.loc[(df.n == n) & (df.Method == m), \"rel_risk\"] = risk[n][obj_coef][m]/risk[n][obj_coef][\"oracle\"]\n",
    "            \n",
    "df = df.loc[df.Method != \"oracle\"]\n",
    "df.loc[df.Method == \"rf_oracle\", \"Constraint\"] = \"yes\"\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Constraint\"] = \"yes\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Constraint\"] = \"yes\"\n",
    "\n",
    "df.loc[df.Method == \"rf_oracle\", \"Method\"] = \"StochOptForest\\n(oracle)\"\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_approx_sol_unconstr\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk_unconstr\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_rf\", \"Method\"] = \"RandForest\"\n",
    "df.loc[df.Method == \"rf_random\", \"Method\"] = \"RandSplitForest\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"cvar/risk_lognormal_oracle.csv\", index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## lognormal combined obj"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "risk = pickle.load(open(\"cvar/risk_cvar_lognormal_objcoef.pkl\", \"rb\"))\n",
    "sample_size = list(risk.keys())\n",
    "obj_coefs = list(risk[sample_size[0]].keys())\n",
    "method = list(risk[sample_size[0]][obj_coefs[0]].keys())\n",
    "runs = len(risk[sample_size[0]][obj_coefs[0]][method[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = {\n",
    "    \"obj_coef\": [key for key in obj_coefs for i in range(runs * len(method))], \n",
    "    \"Method\": [key for key in method for i in range(runs)] * len(obj_coefs),\n",
    "    \"Constraint\": [\"no\"] * len(obj_coefs) * len(method) * runs, \n",
    "    \"rel_risk\": [0] * len(obj_coefs) * len(method) * runs,\n",
    "    \"n\": [\"400\"] * len(obj_coefs) * len(method) * runs\n",
    "}\n",
    "df = pd.DataFrame(data)\n",
    "type_convert = {\n",
    "    \"n\": str,\n",
    "    \"obj_coef\": str,\n",
    "    \"Method\": str,\n",
    "    \"Constraint\": str, \n",
    "    \"rel_risk\": float\n",
    "}\n",
    "df = df.astype(type_convert) \n",
    "\n",
    "for obj_coef in obj_coefs:\n",
    "    for m in method:\n",
    "        df.loc[(df.obj_coef == obj_coef) & (df.Method == m), \"rel_risk\"] = risk[\"400\"][obj_coef][m]/risk[\"400\"][obj_coef][\"oracle\"]\n",
    "\n",
    "df = df.loc[df.Method != \"oracle\"]\n",
    "df.loc[df.Method == \"rf_oracle\", \"Constraint\"] = \"yes\"\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Constraint\"] = \"yes\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Constraint\"] = \"yes\"\n",
    "\n",
    "df.loc[df.Method == \"rf_oracle\", \"Method\"] = \"StochOptForest\\n(oracle)\"\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_approx_sol_unconstr\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk_unconstr\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_rf\", \"Method\"] = \"RandForest\"\n",
    "df.loc[df.Method == \"rf_random\", \"Method\"] = \"RandSplitForest\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"cvar/risk_lognormal_objcoef.csv\", index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## lognormal honesty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "risk = pickle.load(open(\"cvar/risk_cvar_lognormal_honesty.pkl\", \"rb\"))\n",
    "sample_size = list(risk.keys())\n",
    "obj_coefs = list(risk[sample_size[0]].keys())\n",
    "method = list(risk[sample_size[0]][obj_coefs[0]].keys())\n",
    "runs = len(risk[sample_size[0]][obj_coefs[0]][method[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = {\n",
    "    \"n\": [key for key in sample_size for i in range(runs * len(method))],\n",
    "    \"obj_coef\": [\"0\"] * len(sample_size) * len(method) * runs,\n",
    "    \"Method\": [key for key in method for i in range(runs)] * len(sample_size),\n",
    "    \"Honesty\": [\"no\"] * len(sample_size) * len(method) * runs, \n",
    "    \"rel_risk\": [0] * len(sample_size) * len(method) * runs\n",
    "}\n",
    "df = pd.DataFrame(data)\n",
    "type_convert = {\n",
    "    \"n\": str,\n",
    "    \"obj_coef\": str,\n",
    "    \"Method\": str,\n",
    "    \"Honesty\": str, \n",
    "    \"rel_risk\": float\n",
    "}\n",
    "df = df.astype(type_convert) \n",
    "for n in sample_size:\n",
    "    for obj_coef in obj_coefs:\n",
    "        for m in method:\n",
    "            df.loc[(df.n == n) & (df.Method == m), \"rel_risk\"] = risk[n][obj_coef][m]/risk[n][obj_coef][\"oracle\"]\n",
    "            \n",
    "df = df.loc[df.Method != \"oracle\"]\n",
    "df.loc[df.Method == \"rf_approx_sol_honest\", \"Honesty\"] = \"yes\"\n",
    "df.loc[df.Method == \"rf_approx_risk_honest\", \"Honesty\"] = \"yes\"\n",
    "\n",
    "df.loc[df.Method == \"rf_approx_sol_dishonest\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_sol_honest\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "\n",
    "df.loc[df.Method == \"rf_approx_risk_honest\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_approx_risk_dishonest\", \"Method\"] = \"StochOptForest\\n(apx-risk)\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"cvar/risk_lognormal_honesty.csv\", index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## normal "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "risk = pickle.load(open(\"cvar/risk_cvar_normal.pkl\", \"rb\"))\n",
    "sample_size = list(risk.keys())\n",
    "obj_coefs = list(risk[sample_size[0]].keys())\n",
    "method = list(risk[sample_size[0]][obj_coefs[0]].keys())\n",
    "runs = len(risk[sample_size[0]][obj_coefs[0]][method[0]])\n",
    "\n",
    "data = {\n",
    "    \"n\": [key for key in sample_size for i in range(runs * len(method))],\n",
    "    \"obj_coef\": [\"0\"] * len(sample_size) * len(method) * runs,\n",
    "    \"Method\": [key for key in method for i in range(runs)] * len(sample_size),\n",
    "    \"Constraint\": [\"no\"] * len(sample_size) * len(method) * runs, \n",
    "    \"rel_risk\": [0] * len(sample_size) * len(method) * runs\n",
    "}\n",
    "df = pd.DataFrame(data)\n",
    "type_convert = {\n",
    "    \"n\": str,\n",
    "    \"obj_coef\": str,\n",
    "    \"Method\": str,\n",
    "    \"Constraint\": str, \n",
    "    \"rel_risk\": float\n",
    "}\n",
    "df = df.astype(type_convert) \n",
    "for n in sample_size:\n",
    "    for obj_coef in obj_coefs:\n",
    "        for m in method:\n",
    "            df.loc[(df.n == n) & (df.Method == m), \"rel_risk\"] = risk[n][obj_coef][m]/risk[n][obj_coef][\"oracle\"]\n",
    "            \n",
    "df = df.loc[df.Method != \"oracle\"]\n",
    "df.loc[df.Method == \"rf_oracle\", \"Constraint\"] = \"yes\"\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Constraint\"] = \"yes\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Constraint\"] = \"yes\"\n",
    "\n",
    "df.loc[df.Method == \"rf_oracle\", \"Method\"] = \"StochOptForest\\n(oracle)\"\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_approx_sol_unconstr\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk_unconstr\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_rf\", \"Method\"] = \"RandForest\"\n",
    "df.loc[df.Method == \"rf_random\", \"Method\"] = \"RandSplitForest\"\n",
    "\n",
    "df.to_csv(\"cvar/risk_normal.csv\", index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## normal oracle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "risk = pickle.load(open(\"cvar/risk_cvar_normal_oracle.pkl\", \"rb\"))\n",
    "sample_size = list(risk.keys())\n",
    "obj_coefs = list(risk[sample_size[0]].keys())\n",
    "method = list(risk[sample_size[0]][obj_coefs[0]].keys())\n",
    "runs = len(risk[sample_size[0]][obj_coefs[0]][method[0]])\n",
    "\n",
    "data = {\n",
    "    \"n\": [key for key in sample_size for i in range(runs * len(method))],\n",
    "    \"obj_coef\": [\"0\"] * len(sample_size) * len(method) * runs,\n",
    "    \"Method\": [key for key in method for i in range(runs)] * len(sample_size),\n",
    "    \"Constraint\": [\"no\"] * len(sample_size) * len(method) * runs, \n",
    "    \"rel_risk\": [0] * len(sample_size) * len(method) * runs\n",
    "}\n",
    "df = pd.DataFrame(data)\n",
    "type_convert = {\n",
    "    \"n\": str,\n",
    "    \"obj_coef\": str,\n",
    "    \"Method\": str,\n",
    "    \"Constraint\": str, \n",
    "    \"rel_risk\": float\n",
    "}\n",
    "df = df.astype(type_convert) \n",
    "for n in sample_size:\n",
    "    for obj_coef in obj_coefs:\n",
    "        for m in method:\n",
    "            df.loc[(df.n == n) & (df.Method == m), \"rel_risk\"] = risk[n][obj_coef][m]/risk[n][obj_coef][\"oracle\"]\n",
    "            \n",
    "df = df.loc[df.Method != \"oracle\"]\n",
    "df.loc[df.Method == \"rf_oracle\", \"Constraint\"] = \"yes\"\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Constraint\"] = \"yes\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Constraint\"] = \"yes\"\n",
    "\n",
    "df.loc[df.Method == \"rf_oracle\", \"Method\"] = \"StochOptForest\\n(oracle)\"\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_approx_sol_unconstr\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk_unconstr\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_rf\", \"Method\"] = \"RandForest\"\n",
    "df.loc[df.Method == \"rf_random\", \"Method\"] = \"RandSplitForest\"\n",
    "\n",
    "df.to_csv(\"cvar/risk_normal_oracle.csv\", index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Newsvendor "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## sample size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "risk = pickle.load(open(\"newsvendor/risk_n.pkl\", \"rb\"))\n",
    "sample_size = list(risk.keys())\n",
    "ps = list(risk[sample_size[0]].keys())\n",
    "method = list(risk[sample_size[0]][ps[0]].keys())\n",
    "runs = len(risk[sample_size[0]][ps[0]][method[0]])\n",
    "\n",
    "data = {\n",
    "    \"n\": [key for key in sample_size for i in range(runs * len(method))],\n",
    "    \"Method\": [key for key in method for i in range(runs)] * len(sample_size),\n",
    "    \"rel_risk\": [0] * len(sample_size) * len(method) * runs,\n",
    "    \"p\": [10] * len(sample_size) * len(method) * runs\n",
    "}\n",
    "df = pd.DataFrame(data)\n",
    "type_convert = {\n",
    "    \"n\": str,\n",
    "    \"Method\": str,\n",
    "    \"rel_risk\": float,\n",
    "    \"p\": str\n",
    "}\n",
    "df = df.astype(type_convert) \n",
    "\n",
    "for n in sample_size:\n",
    "    for p in ps:\n",
    "        for m in method:\n",
    "            df.loc[(df.n == n) & (df.Method == m) & (df.p == p), \"rel_risk\"] = (risk[n][p][m])/risk[n][p][\"oracle\"]\n",
    "            \n",
    "df = df[df.Method != \"oracle\"]\n",
    "df.loc[df.Method == \"rf_oracle\", \"Method\"] = \"StochOptForest\\n(oracle)\"\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_rf\", \"Method\"] = \"RandForest\"\n",
    "df.loc[df.Method == \"rf_grf\", \"Method\"] = \"GenRandForest\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Method</th>\n",
       "      <th>GenRandForest</th>\n",
       "      <th>RandForest</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>1.102898</td>\n",
       "      <td>1.115832</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>200</th>\n",
       "      <td>1.117377</td>\n",
       "      <td>1.117858</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>400</th>\n",
       "      <td>1.085524</td>\n",
       "      <td>1.087415</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>800</th>\n",
       "      <td>1.067499</td>\n",
       "      <td>1.076195</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Method  GenRandForest  RandForest\n",
       "n                                \n",
       "100          1.102898    1.115832\n",
       "200          1.117377    1.117858\n",
       "400          1.085524    1.087415\n",
       "800          1.067499    1.076195"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meanrisks=df.groupby(['n','Method']).agg(mean_risk=pd.NamedAgg(column='rel_risk', aggfunc=np.mean)).reset_index().pivot(index='n', columns='Method', values='mean_risk')\n",
    "meanrisks[['GenRandForest','RandForest']].div(meanrisks['StochOptForest\\n(apx-soln)'], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"newsvendor/risk_nv_n.csv\", index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## feature splititng "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_split = pickle.load(open(\"newsvendor/feature_split_n.pkl\", \"rb\"))\n",
    "feature_split = {\n",
    "    n: {method: feature_split[n][\"10\"][method].mean(0) for method in feature_split[n][\"10\"].keys()} for n in feature_split.keys()\n",
    "}\n",
    "feature_split = feature_split[\"400\"]\n",
    "\n",
    "method = list(feature_split.keys())\n",
    "dimension = len(feature_split[method[0]])\n",
    "\n",
    "data = {\n",
    "    \"Method\": [m for m in method for i in range(dimension)],\n",
    "    \"p\": [str(d) for d in list(range(1, dimension+1))] * len(method),\n",
    "    \"freq\": [0] * len(method) * dimension\n",
    "}\n",
    "df = pd.DataFrame(data)\n",
    "type_convert = {\n",
    "    \"Method\": str,\n",
    "    \"p\": str,\n",
    "    \"freq\": float\n",
    "}\n",
    "\n",
    "df = df.astype(type_convert) \n",
    "\n",
    "for m in method:\n",
    "    for d in list(range(1, dimension + 1)):\n",
    "        df.loc[(df.Method == m) & (df.p == str(d)), \"freq\"] = feature_split[m][d-1]\n",
    "        \n",
    "df = df[df.Method != \"oracle\"]\n",
    "df.loc[df.Method == \"rf_oracle\", \"Method\"] = \"StochOptForest\\n(oracle)\"\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_rf\", \"Method\"] = \"RandForest\"\n",
    "df.loc[df.Method == \"rf_grf\", \"Method\"] = \"GenRandForest\"\n",
    "\n",
    "df.to_csv(\"newsvendor/feature_split_nv_n.csv\", index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature Importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"newsvendor/feature_importance_n.pkl\", 'rb') as pickle_file:\n",
    "    fi = pickle.load(pickle_file)\n",
    "fi = fi[\"400\"][\"10\"]\n",
    "mean_imp = {key: fi[key].mean(0) for key in fi.keys()}\n",
    "pd.DataFrame({key: (mean_imp[key] - np.min(mean_imp[key]))/(np.max(mean_imp[key]) - np.min(mean_imp[key])) \n",
    "              for key in mean_imp.keys()}).to_csv(\"newsvendor/feature_importance_n.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## dimensionality "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "risk = pickle.load(open(\"newsvendor/risk_p.pkl\", \"rb\"))\n",
    "\n",
    "sample_size = list(risk.keys())\n",
    "dimension = list(risk[sample_size[0]].keys())\n",
    "method = list(risk[sample_size[0]][dimension[0]].keys())\n",
    "runs = len(risk[sample_size[0]][dimension[0]][method[0]])\n",
    "\n",
    "data = {\n",
    "    \"p\": [key for key in dimension for i in range(runs * len(method))],\n",
    "    \"Method\": [key for key in method for i in range(runs)] * len(dimension),\n",
    "    \"rel_risk\": [0] * len(dimension) * len(method) * runs\n",
    "}\n",
    "df = pd.DataFrame(data)\n",
    "type_convert = {\n",
    "    \"p\": str,\n",
    "    \"Method\": str,\n",
    "    \"rel_risk\": float\n",
    "}\n",
    "df = df.astype(type_convert) \n",
    "\n",
    "for d in dimension:\n",
    "    for m in method:\n",
    "        df.loc[(df.p == d) & (df.Method == m), \"rel_risk\"] = risk['800'][d][m]/risk['800'][d]['oracle']\n",
    "        \n",
    "df = df[df.Method != \"oracle\"]\n",
    "df = df[df.Method != \"SAA\"]\n",
    "df.loc[df.Method == \"rf_oracle\", \"Method\"] = \"StochOptForest\\n(oracle)\"\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_rf\", \"Method\"] = \"RandForest\"\n",
    "df.loc[df.Method == \"rf_grf\", \"Method\"] = \"GenRandForest\"\n",
    "df.loc[df.Method == \"rf_random\", \"Method\"] = \"RandSplitForest\"\n",
    "df.loc[df.Method == \"knn_5\", \"Method\"] = \"5-NN\"\n",
    "df.loc[df.Method == \"knn_10\", \"Method\"] = \"10-NN\"\n",
    "df.loc[df.Method == \"knn_50\", \"Method\"] = \"50-NN\"\n",
    "\n",
    "df.to_csv(\"newsvendor/risk_nv_p.csv\", index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## High Dimension"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "risk = pickle.load(open(\"newsvendor/risk_highdim.pkl\", \"rb\"))\n",
    "\n",
    "sample_size = list(risk.keys())\n",
    "dimension = list(risk[sample_size[0]].keys())\n",
    "method = list(risk[sample_size[0]][dimension[0]].keys())\n",
    "runs = len(risk[sample_size[0]][dimension[0]][method[0]])\n",
    "\n",
    "data = {\n",
    "    \"p\": [key for key in dimension for i in range(runs * len(method))],\n",
    "    \"Method\": [key for key in method for i in range(runs)] * len(dimension),\n",
    "    \"rel_risk\": [0] * len(dimension) * len(method) * runs\n",
    "}\n",
    "df = pd.DataFrame(data)\n",
    "type_convert = {\n",
    "    \"p\": str,\n",
    "    \"Method\": str,\n",
    "    \"rel_risk\": float\n",
    "}\n",
    "df = df.astype(type_convert) \n",
    "\n",
    "for d in dimension:\n",
    "    for m in method:\n",
    "        df.loc[(df.p == d) & (df.Method == m), \"rel_risk\"] = risk['200'][d][m]/risk['200'][d]['oracle']\n",
    "        \n",
    "df = df[df.Method != \"oracle\"]\n",
    "df = df[df.Method != \"SAA\"]\n",
    "df.loc[df.Method == \"rf_oracle\", \"Method\"] = \"StochOptForest\\n(oracle)\"\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_rf\", \"Method\"] = \"RandForest\"\n",
    "df.loc[df.Method == \"rf_grf\", \"Method\"] = \"GenRandForest\"\n",
    "df.loc[df.Method == \"rf_random\", \"Method\"] = \"RandSplitForest\"\n",
    "df.loc[df.Method == \"knn_5\", \"Method\"] = \"5-NN\"\n",
    "df.loc[df.Method == \"knn_10\", \"Method\"] = \"10-NN\"\n",
    "df.loc[df.Method == \"knn_50\", \"Method\"] = \"50-NN\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"newsvendor/risk_nv_highdim.csv\", index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Honesty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "risk = pickle.load(open(\"newsvendor/risk_nv_honesty.pkl\", \"rb\"))\n",
    "sample_size = list(risk.keys())\n",
    "obj_coefs = list(risk[sample_size[0]].keys())\n",
    "method = list(risk[sample_size[0]][obj_coefs[0]].keys())\n",
    "runs = len(risk[sample_size[0]][obj_coefs[0]][method[0]])\n",
    "\n",
    "data = {\n",
    "    \"n\": [key for key in sample_size for i in range(runs * len(method))],\n",
    "    \"obj_coef\": [\"0\"] * len(sample_size) * len(method) * runs,\n",
    "    \"Method\": [key for key in method for i in range(runs)] * len(sample_size),\n",
    "    \"Honesty\": [\"no\"] * len(sample_size) * len(method) * runs, \n",
    "    \"rel_risk\": [0] * len(sample_size) * len(method) * runs\n",
    "}\n",
    "df = pd.DataFrame(data)\n",
    "type_convert = {\n",
    "    \"n\": str,\n",
    "    \"obj_coef\": str,\n",
    "    \"Method\": str,\n",
    "    \"Honesty\": str, \n",
    "    \"rel_risk\": float\n",
    "}\n",
    "df = df.astype(type_convert) \n",
    "for n in sample_size:\n",
    "    for obj_coef in obj_coefs:\n",
    "        for m in method:\n",
    "            df.loc[(df.n == n) & (df.Method == m), \"rel_risk\"] = risk[n][obj_coef][m]/risk[n][obj_coef][\"oracle\"]\n",
    "            \n",
    "df = df.loc[df.Method != \"oracle\"]\n",
    "df.loc[df.Method == \"rf_approx_sol_honest\", \"Honesty\"] = \"yes\"\n",
    "df.loc[df.Method == \"rf_approx_risk_honest\", \"Honesty\"] = \"yes\"\n",
    "\n",
    "df.loc[df.Method == \"rf_approx_sol_dishonest\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_sol_honest\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "\n",
    "df.loc[df.Method == \"rf_approx_risk_honest\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_approx_risk_dishonest\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "\n",
    "df.to_csv(\"newsvendor/risk_nv_honesty.csv\", index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Mean-variance Optimization\n",
    "\n",
    "## risk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "risk = pickle.load(open(\"mean_var/rel_risk_meanvar_normal_stoch.pkl\", \"rb\"))\n",
    "sample_size = list(risk.keys())\n",
    "Rs = list(risk[sample_size[0]].keys())\n",
    "method = list(risk[sample_size[0]][Rs[0]].keys())\n",
    "runs = len(risk[sample_size[0]][Rs[0]][method[0]])\n",
    "\n",
    "data = {\n",
    "    \"n\": [key for key in sample_size for i in range(runs * len(method))],\n",
    "    \"R\": [\"0.5\"] * len(sample_size) * len(method) * runs,\n",
    "    \"Method\": [key for key in method for i in range(runs)] * len(sample_size),\n",
    "    \"Constraint\": [\"no\"] * len(sample_size) * len(method) * runs, \n",
    "    \"rel_risk\": [0] * len(sample_size) * len(method) * runs\n",
    "}\n",
    "df = pd.DataFrame(data)\n",
    "type_convert = {\n",
    "    \"n\": str,\n",
    "    \"R\": str,\n",
    "    \"Method\": str,\n",
    "    \"Constraint\": str, \n",
    "    \"rel_risk\": float\n",
    "}\n",
    "df = df.astype(type_convert) \n",
    "for n in sample_size:\n",
    "    for R in Rs:\n",
    "        for m in method:\n",
    "            df.loc[(df.n == n) & (df.Method == m), \"rel_risk\"] = risk[n][R][m]\n",
    "            \n",
    "df = df.loc[df.Method != \"oracle\"]\n",
    "df = df.loc[df.Method != \"rf_oracle\"]\n",
    "df.loc[df.Method == \"rf_oracle\", \"Constraint\"] = \"yes\"\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Constraint\"] = \"yes\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Constraint\"] = \"yes\"\n",
    "\n",
    "# df.loc[df.Method == \"rf_oracle\", \"Method\"] = \"StochOptForest\\n(oracle)\"\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_approx_sol_unconstr\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk_unconstr\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_rf\", \"Method\"] = \"RandForest\"\n",
    "df.loc[df.Method == \"rf_random\", \"Method\"] = \"RandSplitForest\"\n",
    "\n",
    "df['n'] = pd.Categorical(df['n'], categories=['100', '200', '400', '800'])\n",
    "df['Method'] = pd.Categorical(df['Method'], categories=['StochOptForest\\n(apx-soln)', 'StochOptForest\\n(apx-risk)',  'RandForest', 'RandSplitForest'])\n",
    "df['Constraint'] = pd.Categorical(df['Constraint'], categories = [\"yes\", \"no\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"mean_var/rel_risk_full.csv\", index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## feature splitting "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_split = pickle.load(open(\"mean_var/feature_split_meanvar_normal_stoch.pkl\", \"rb\"))\n",
    "feature_split = feature_split[\"800\"][\"0.5\"]\n",
    "feature_split = {\n",
    "    method: feature_split[method].mean(0) for method in feature_split.keys()\n",
    "}\n",
    "method = list(feature_split.keys())\n",
    "dimension = len(feature_split[method[0]])\n",
    "\n",
    "data = {\n",
    "    \"Method\": [m for m in method for i in range(dimension)],\n",
    "    \"p\": [str(d) for d in list(range(1, 11))] * len(method),\n",
    "    \"Constraint\": [\"no\"] * len(method) * dimension, \n",
    "    \"freq\": [0] * len(method) * dimension\n",
    "}\n",
    "df = pd.DataFrame(data)\n",
    "type_convert = {\n",
    "    \"Method\": str,\n",
    "    \"p\": str,\n",
    "    \"Constraint\": str,\n",
    "    \"freq\": float\n",
    "}\n",
    "df = df.astype(type_convert) \n",
    "\n",
    "for m in method:\n",
    "    for d in list(range(1, dimension + 1)):\n",
    "        df.loc[(df.Method == m) & (df.p == str(d)), \"freq\"] = feature_split[m][d-1]\n",
    "        \n",
    "df = df.loc[df.Method != \"oracle\"]\n",
    "df = df.loc[df.Method != \"rf_oracle\"]\n",
    "df.loc[df.Method == \"rf_oracle\", \"Constraint\"] = \"yes\"\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Constraint\"] = \"yes\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Constraint\"] = \"yes\"\n",
    "\n",
    "# df.loc[df.Method == \"rf_oracle\", \"Method\"] = \"StochOptForest\\n(oracle)\"\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_approx_sol_unconstr\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk_unconstr\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_rf\", \"Method\"] = \"RandForest\"\n",
    "df.loc[df.Method == \"rf_random\", \"Method\"] = \"RandSplitForest\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"mean_var/feature_freq_full.csv\", index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## conditional constraint violation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "violation = pickle.load(open(\"mean_var/cond_violation_meanvar_normal_stoch.pkl\", \"rb\"))\n",
    "sample_size = list(violation.keys())\n",
    "Rs = list(violation[sample_size[0]].keys())\n",
    "method = list(violation[sample_size[0]][Rs[0]].keys())\n",
    "runs = len(violation[sample_size[0]][Rs[0]][method[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = {\n",
    "    \"n\": [key for key in sample_size for i in range(runs * len(method))],\n",
    "    \"R\": [\"0.5\"] * len(sample_size) * len(method) * runs,\n",
    "    \"Method\": [key for key in method for i in range(runs)] * len(sample_size),\n",
    "    \"Constraint\": [\"no\"] * len(sample_size) * len(method) * runs, \n",
    "    \"violation\": [0] * len(sample_size) * len(method) * runs\n",
    "}\n",
    "df = pd.DataFrame(data)\n",
    "type_convert = {\n",
    "    \"n\": str,\n",
    "    \"R\": str,\n",
    "    \"Method\": str,\n",
    "    \"Constraint\": str, \n",
    "    \"violation\": float\n",
    "}\n",
    "df = df.astype(type_convert) \n",
    "for n in sample_size:\n",
    "    for R in Rs:\n",
    "        for m in method:\n",
    "            df.loc[(df.n == n) & (df.Method == m), \"violation\"] = violation[n][R][m]\n",
    "            \n",
    "df = df.loc[df.Method != \"oracle\"]\n",
    "df = df.loc[df.Method != \"rf_oracle\"]\n",
    "df = df.loc[df.Method != \"rf_random_oracle\"]\n",
    "df = df.loc[df.Method != \"rf_approx_risk_oracle\"]\n",
    "df = df.loc[df.Method != \"rf_approx_risk_unconstr_oracle\"]\n",
    "df = df.loc[df.Method != \"rf_approx_sol_oracle\"]\n",
    "df = df.loc[df.Method != \"rf_approx_sol_unconstr_oracle\"]\n",
    "df = df.loc[df.Method != \"rf_rf_oracle\"]\n",
    "df = df.loc[df.Method != \"rf_oracle_oracle\"]\n",
    "\n",
    "# df.loc[df.Method == \"rf_oracle\", \"Constraint\"] = \"yes\"\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Constraint\"] = \"yes\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Constraint\"] = \"yes\"\n",
    "\n",
    "# df.loc[df.Method == \"rf_oracle\", \"Method\"] = \"StochOptForest\\n(oracle)\"\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_approx_sol_unconstr\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk_unconstr\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_rf\", \"Method\"] = \"RandForest\"\n",
    "df.loc[df.Method == \"rf_random\", \"Method\"] = \"RandSplitForest\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"mean_var/cond_violation_full.csv\", index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## marginal constraint violation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "violation = pickle.load(open(\"mean_var/mean_violation_meanvar_normal_stoch.pkl\", \"rb\"))\n",
    "sample_size = list(violation.keys())\n",
    "Rs = list(violation[sample_size[0]].keys())\n",
    "method = list(violation[sample_size[0]][Rs[0]].keys())\n",
    "runs = len(violation[sample_size[0]][Rs[0]][method[0]])\n",
    "\n",
    "data = {\n",
    "    \"n\": [key for key in sample_size for i in range(runs * len(method))],\n",
    "    \"R\": [\"0.5\"] * len(sample_size) * len(method) * runs,\n",
    "    \"Method\": [key for key in method for i in range(runs)] * len(sample_size),\n",
    "    \"Constraint\": [\"no\"] * len(sample_size) * len(method) * runs, \n",
    "    \"violation\": [0] * len(sample_size) * len(method) * runs\n",
    "}\n",
    "df = pd.DataFrame(data)\n",
    "type_convert = {\n",
    "    \"n\": str,\n",
    "    \"R\": str,\n",
    "    \"Method\": str,\n",
    "    \"Constraint\": str, \n",
    "    \"violation\": float\n",
    "}\n",
    "df = df.astype(type_convert) \n",
    "for n in sample_size:\n",
    "    for R in Rs:\n",
    "        for m in method:\n",
    "            df.loc[(df.n == n) & (df.Method == m), \"violation\"] = violation[n][R][m]\n",
    "            \n",
    "df = df.loc[df.Method != \"oracle\"]\n",
    "df = df.loc[df.Method != \"rf_oracle\"]\n",
    "df = df.loc[df.Method != \"rf_random_oracle\"]\n",
    "df = df.loc[df.Method != \"rf_approx_risk_oracle\"]\n",
    "df = df.loc[df.Method != \"rf_approx_risk_unconstr_oracle\"]\n",
    "df = df.loc[df.Method != \"rf_approx_sol_oracle\"]\n",
    "df = df.loc[df.Method != \"rf_approx_sol_unconstr_oracle\"]\n",
    "df = df.loc[df.Method != \"rf_rf_oracle\"]\n",
    "df = df.loc[df.Method != \"rf_oracle_oracle\"]\n",
    "\n",
    "# df.loc[df.Method == \"rf_oracle\", \"Constraint\"] = \"yes\"\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Constraint\"] = \"yes\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Constraint\"] = \"yes\"\n",
    "\n",
    "# df.loc[df.Method == \"rf_oracle\", \"Method\"] = \"StochOptForest\\n(oracle)\"\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_approx_sol_unconstr\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk_unconstr\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_rf\", \"Method\"] = \"RandForest\"\n",
    "df.loc[df.Method == \"rf_random\", \"Method\"] = \"RandSplitForest\"\n",
    "\n",
    "df['n'] = pd.Categorical(df['n'], categories=['100', '200', '400', \"800\"])\n",
    "df['Method'] = pd.Categorical(df['Method'], categories=['StochOptForest\\n(apx-soln)', 'StochOptForest\\n(apx-risk)', 'RandForest', 'RandSplitForest'])\n",
    "df['Constraint'] = pd.Categorical(df['Constraint'], categories = [\"yes\", \"no\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"mean_var/marginal_violation_full.csv\", index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## oracle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "risk = pickle.load(open(\"mean_var/rel_risk_meanvar_normal_stoch_oracle.pkl\", \"rb\"))\n",
    "sample_size = list(risk.keys())\n",
    "Rs = list(risk[sample_size[0]].keys())\n",
    "method = list(risk[sample_size[0]][Rs[0]].keys())\n",
    "runs = len(risk[sample_size[0]][Rs[0]][method[0]])\n",
    "\n",
    "data = {\n",
    "    \"n\": [key for key in sample_size for i in range(runs * len(method))],\n",
    "    \"R\": [\"0.1\"] * len(sample_size) * len(method) * runs,\n",
    "    \"Method\": [key for key in method for i in range(runs)] * len(sample_size),\n",
    "    \"Constraint\": [\"no\"] * len(sample_size) * len(method) * runs, \n",
    "    \"rel_risk\": [0] * len(sample_size) * len(method) * runs\n",
    "}\n",
    "df = pd.DataFrame(data)\n",
    "type_convert = {\n",
    "    \"n\": str,\n",
    "    \"R\": str,\n",
    "    \"Method\": str,\n",
    "    \"Constraint\": str, \n",
    "    \"rel_risk\": float\n",
    "}\n",
    "df = df.astype(type_convert) \n",
    "for n in sample_size:\n",
    "    for R in Rs:\n",
    "        for m in method:\n",
    "            df.loc[(df.n == n) & (df.Method == m), \"rel_risk\"] = risk[n][R][m]\n",
    "            \n",
    "df = df.loc[df.Method != \"oracle\"]\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Constraint\"] = \"yes\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Constraint\"] = \"yes\"\n",
    "df.loc[df.Method == \"rf_oracle\", \"Constraint\"] = \"yes\"\n",
    "\n",
    "df.loc[df.Method == \"rf_oracle\", \"Method\"] = \"StochOptForest\\n(oracle)\"\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_approx_sol_unconstr\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk_unconstr\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_rf\", \"Method\"] = \"RandForest\"\n",
    "df.loc[df.Method == \"rf_random\", \"Method\"] = \"RandSplitForest\"\n",
    "\n",
    "df['n'] = pd.Categorical(df['n'], categories=['100', '200', '400', '800'])\n",
    "df['Method'] = pd.Categorical(df['Method'], categories=['StochOptForest\\n(oracle)', 'StochOptForest\\n(apx-soln)', 'StochOptForest\\n(apx-risk)', 'RandForest', 'RandSplitForest'])\n",
    "df['Constraint'] = pd.Categorical(df['Constraint'], categories = [\"yes\", \"no\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"mean_var/rel_risk_full_oracle.csv\", index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## oracle feature splitting "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_split = pickle.load(open(\"mean_var/feature_split_meanvar_normal_stoch_oracle.pkl\", \"rb\"))\n",
    "feature_split = feature_split[\"400\"][\"0.1\"]\n",
    "feature_split = {\n",
    "    method: feature_split[method].mean(0) for method in feature_split.keys()\n",
    "}\n",
    "method = list(feature_split.keys())\n",
    "dimension = len(feature_split[method[0]])\n",
    "\n",
    "data = {\n",
    "    \"Method\": [m for m in method for i in range(dimension)],\n",
    "    \"p\": [str(d) for d in list(range(1, 11))] * len(method),\n",
    "    \"Constraint\": [\"no\"] * len(method) * dimension, \n",
    "    \"freq\": [0] * len(method) * dimension\n",
    "}\n",
    "df = pd.DataFrame(data)\n",
    "type_convert = {\n",
    "    \"Method\": str,\n",
    "    \"p\": str,\n",
    "    \"Constraint\": str,\n",
    "    \"freq\": float\n",
    "}\n",
    "df = df.astype(type_convert) \n",
    "\n",
    "for m in method:\n",
    "    for d in list(range(1, dimension + 1)):\n",
    "        df.loc[(df.Method == m) & (df.p == str(d)), \"freq\"] = feature_split[m][d-1]\n",
    "        \n",
    "df = df.loc[df.Method != \"oracle\"]\n",
    "# df = df.loc[df.Method != \"rf_oracle\"]\n",
    "df = df.loc[df.Method != \"rf_random\"]\n",
    "df.loc[df.Method == \"rf_oracle\", \"Constraint\"] = \"yes\"\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Constraint\"] = \"yes\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Constraint\"] = \"yes\"\n",
    "\n",
    "df.loc[df.Method == \"rf_oracle\", \"Method\"] = \"StochOptForest\\n(oracle)\"\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_approx_sol_unconstr\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk_unconstr\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_rf\", \"Method\"] = \"RandForest\"\n",
    "# df.loc[df.Method == \"rf_random\", \"Method\"] = \"RandSplitForest\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"mean_var/feature_freq_full_oracle.csv\", index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## constraint threshold "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "risk = pickle.load(open(\"mean_var/rel_risk_meanvar_normal_stoch_R.pkl\", \"rb\"))\n",
    "sample_size = list(risk.keys())\n",
    "Rs = list(risk[sample_size[0]].keys())\n",
    "method = list(risk[sample_size[0]][Rs[0]].keys())\n",
    "runs = len(risk[sample_size[0]][Rs[0]][method[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = {\n",
    "    \"R\":  [key for key in Rs for i in range(runs * len(method))],\n",
    "    \"Method\": [key for key in method for i in range(runs)] * len(Rs),\n",
    "    \"Constraint\": [\"no\"] * len(Rs) * len(method) * runs, \n",
    "    \"rel_risk\": [0] * len(Rs) * len(method) * runs,\n",
    "    \"n\": [\"400\"] * len(Rs) * len(method) * runs\n",
    "}\n",
    "df = pd.DataFrame(data)\n",
    "type_convert = {\n",
    "    \"n\": str,\n",
    "    \"R\": str,\n",
    "    \"Method\": str,\n",
    "    \"Constraint\": str, \n",
    "    \"rel_risk\": float\n",
    "}\n",
    "df = df.astype(type_convert) \n",
    "\n",
    "for R in Rs:\n",
    "    for m in method:\n",
    "        df.loc[(df.R == R) & (df.Method == m), \"rel_risk\"] = risk[\"400\"][R][m]          \n",
    "    \n",
    "df = df.loc[df.Method != \"oracle\"]\n",
    "df = df.loc[df.Method != \"rf_oracle\"]\n",
    "df.loc[df.Method == \"rf_oracle\", \"Constraint\"] = \"yes\"\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Constraint\"] = \"yes\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Constraint\"] = \"yes\"\n",
    "\n",
    "# df.loc[df.Method == \"rf_oracle\", \"Method\"] = \"StochOptForest\\n(oracle)\"\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_approx_sol_unconstr\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk_unconstr\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_rf\", \"Method\"] = \"RandForest\"\n",
    "df.loc[df.Method == \"rf_random\", \"Method\"] = \"RandSplitForest\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"mean_var/rel_risk_full_R.csv\", index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# minimum-variance optimization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## risk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "risk = pickle.load(open(\"mean_var/risk_var_normal.pkl\", \"rb\"))\n",
    "\n",
    "sample_size = list(risk.keys())\n",
    "obj_coefs = list(risk[sample_size[0]].keys())\n",
    "method = list(risk[sample_size[0]][obj_coefs[0]].keys())\n",
    "runs = len(risk[sample_size[0]][obj_coefs[0]][method[0]])\n",
    "\n",
    "data = {\n",
    "    \"n\": [key for key in sample_size for i in range(runs * len(method))],\n",
    "    \"obj_coef\": [\"0\"] * len(sample_size) * len(method) * runs,\n",
    "    \"Method\": [key for key in method for i in range(runs)] * len(sample_size),\n",
    "    \"Constraint\": [\"no\"] * len(sample_size) * len(method) * runs, \n",
    "    \"rel_risk\": [0] * len(sample_size) * len(method) * runs\n",
    "}\n",
    "df = pd.DataFrame(data)\n",
    "type_convert = {\n",
    "    \"n\": str,\n",
    "    \"obj_coef\": str,\n",
    "    \"Method\": str,\n",
    "    \"Constraint\": str, \n",
    "    \"rel_risk\": float\n",
    "}\n",
    "df = df.astype(type_convert) \n",
    "for n in sample_size:\n",
    "    for obj_coef in obj_coefs:\n",
    "        for m in method:\n",
    "            df.loc[(df.n == n) & (df.Method == m), \"rel_risk\"] = risk[n][obj_coef][m]/risk[n][obj_coef][\"oracle\"]\n",
    "            \n",
    "df = df.loc[df.Method != \"oracle\"]\n",
    "df.loc[df.Method == \"rf_oracle\", \"Constraint\"] = \"yes\"\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Constraint\"] = \"yes\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Constraint\"] = \"yes\"\n",
    "\n",
    "df.loc[df.Method == \"rf_oracle\", \"Method\"] = \"StochOptForest\\n(oracle)\"\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_approx_sol_unconstr\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk_unconstr\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_rf\", \"Method\"] = \"RandForest\"\n",
    "df.loc[df.Method == \"rf_random\", \"Method\"] = \"RandSplitForest\"\n",
    "\n",
    "df['n'] = pd.Categorical(df['n'], categories=['100', '200', '400', '800'])\n",
    "df['Method'] = pd.Categorical(df['Method'], categories=['StochOptForest\\n(oracle)', 'StochOptForest\\n(apx-soln)', 'StochOptForest\\n(apx-risk)', 'RandForest', 'RandSplitForest'])\n",
    "df['Constraint'] = pd.Categorical(df['Constraint'], categories = [\"yes\", \"no\"])\n",
    "\n",
    "df.to_csv(\"mean_var/risk_var_normal.csv\", index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## oracle "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "risk = pickle.load(open(\"mean_var/risk_var_normal_oracle.pkl\", \"rb\"))\n",
    "\n",
    "sample_size = list(risk.keys())\n",
    "obj_coefs = list(risk[sample_size[0]].keys())\n",
    "method = list(risk[sample_size[0]][obj_coefs[0]].keys())\n",
    "runs = len(risk[sample_size[0]][obj_coefs[0]][method[0]])\n",
    "\n",
    "data = {\n",
    "    \"n\": [key for key in sample_size for i in range(runs * len(method))],\n",
    "    \"obj_coef\": [\"0\"] * len(sample_size) * len(method) * runs,\n",
    "    \"Method\": [key for key in method for i in range(runs)] * len(sample_size),\n",
    "    \"Constraint\": [\"no\"] * len(sample_size) * len(method) * runs, \n",
    "    \"rel_risk\": [0] * len(sample_size) * len(method) * runs\n",
    "}\n",
    "df = pd.DataFrame(data)\n",
    "type_convert = {\n",
    "    \"n\": str,\n",
    "    \"obj_coef\": str,\n",
    "    \"Method\": str,\n",
    "    \"Constraint\": str, \n",
    "    \"rel_risk\": float\n",
    "}\n",
    "df = df.astype(type_convert) \n",
    "for n in sample_size:\n",
    "    for obj_coef in obj_coefs:\n",
    "        for m in method:\n",
    "            df.loc[(df.n == n) & (df.Method == m), \"rel_risk\"] = risk[n][obj_coef][m]/risk[n][obj_coef][\"oracle\"]\n",
    "            \n",
    "df = df.loc[df.Method != \"oracle\"]\n",
    "df.loc[df.Method == \"rf_oracle\", \"Constraint\"] = \"yes\"\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Constraint\"] = \"yes\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Constraint\"] = \"yes\"\n",
    "\n",
    "df.loc[df.Method == \"rf_oracle\", \"Method\"] = \"StochOptForest\\n(oracle)\"\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_approx_sol_unconstr\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk_unconstr\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_rf\", \"Method\"] = \"RandForest\"\n",
    "df.loc[df.Method == \"rf_random\", \"Method\"] = \"RandSplitForest\"\n",
    "\n",
    "df['n'] = pd.Categorical(df['n'], categories=['100', '200', '400', '800'])\n",
    "df['Method'] = pd.Categorical(df['Method'], categories=['StochOptForest\\n(oracle)', 'StochOptForest\\n(apx-soln)', 'StochOptForest\\n(apx-risk)', 'RandForest', 'RandSplitForest'])\n",
    "df['Constraint'] = pd.Categorical(df['Constraint'], categories = [\"yes\", \"no\"])\n",
    "\n",
    "df.to_csv(\"mean_var/risk_var_normal_oracle.csv\", index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## oracle feature splitting "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_split = pickle.load(open(\"mean_var/feature_split_var_normal_oracle.pkl\", \"rb\"))\n",
    "feature_split = feature_split[\"400\"][\"0\"]\n",
    "feature_split = {\n",
    "    method: feature_split[method].mean(0) for method in feature_split.keys()\n",
    "}\n",
    "\n",
    "method = list(feature_split.keys())\n",
    "dimension = len(feature_split[method[0]])\n",
    "\n",
    "data = {\n",
    "    \"Method\": [m for m in method for i in range(dimension)],\n",
    "    \"p\": [str(d) for d in list(range(1, 11))] * len(method),\n",
    "    \"Constraint\": [\"no\"] * len(method) * dimension, \n",
    "    \"freq\": [0] * len(method) * dimension\n",
    "}\n",
    "df = pd.DataFrame(data)\n",
    "type_convert = {\n",
    "    \"Method\": str,\n",
    "    \"p\": str,\n",
    "    \"Constraint\": str,\n",
    "    \"freq\": float\n",
    "}\n",
    "df = df.astype(type_convert) \n",
    "\n",
    "for m in method:\n",
    "    for d in list(range(1, dimension + 1)):\n",
    "        df.loc[(df.Method == m) & (df.p == str(d)), \"freq\"] = feature_split[m][d-1]\n",
    "        \n",
    "df = df.loc[df.Method != \"rf_random\"]\n",
    "\n",
    "df.loc[df.Method == \"rf_oracle\", \"Constraint\"] = \"yes\"\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Constraint\"] = \"yes\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Constraint\"] = \"yes\"\n",
    "\n",
    "df.loc[df.Method == \"rf_oracle\", \"Method\"] = \"StochOptForest\\n(oracle)\"\n",
    "df.loc[df.Method == \"rf_approx_sol\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_approx_sol_unconstr\", \"Method\"] = \"StochOptForest\\n(apx-soln)\"\n",
    "df.loc[df.Method == \"rf_approx_risk_unconstr\", \"Method\"] = \"StochOptForest\\n(apx-risk)\"\n",
    "df.loc[df.Method == \"rf_rf\", \"Method\"] = \"RandForest\"\n",
    "\n",
    "df.to_csv(\"mean_var/feature_split_var_normal_oracle.csv\", index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Uber Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "for year in [\"twoyear\",\"oneyear\",\"halfyear\",\"onehalfyear\"]:\n",
    "    with open(\"uber/downtown_risks_forest_years_\"+year+\".pkl\", 'rb') as pickle_file:\n",
    "        risk = pickle.load(pickle_file)\n",
    "        risk.to_csv(\"uber/downtown_risks_forest_years_\"+year+\".csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
